#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
copyright (c) 2009, paketka@gmail.com et. al
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.

    * Neither the name of the <ORGANIZATION> nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from django.core.management import setup_environ
import settings
setup_environ(settings)
from django.core.files import File
from django.core.exceptions import ObjectDoesNotExist
from django.db import IntegrityError

import sys
import Image
import httplib
import os.path
import utils.fetcher
import unicodedata
import re
import datetime
from psp.models import *

def fixNames():
    for mp in Person.objects.all():
        if not mp.name.istitle():
            old = '%s %s' % (mp.name, mp.surname)
            mp.title += ' ' + mp.name
            list = mp.surname.split()
            mp.name = list.pop(0)
            newSurname = ''
            for n in list:
                newSurname += n + ' '
            mp.surname = newSurname.strip()
            mp.save()
            print 'Fixed: %s to %s %s' % (old, mp.name, mp.surname)


def fixOrgs():
    """
    fixes group names. turns origName into name and find proper
    parent group for newly created subgroup.
    """
    splitPattern = re.compile(u'(?P<boardName>.*)(?P<subBoardName>[Pp]odvýbor.*)', re.U | re.I)

    for g in Group.objects.all():
        g.name = g.origName.strip()
        g.type = g.type.strip().upper()
        g.save()

    for g in Group.objects.filter(origName__iregex = u'podvýbor'):
        match = splitPattern.match(g.origName)
        if match != None:
            boardName = match.group('boardName').strip()
            g.name = match.group('subBoardName').strip()

            try:
                pg = Group.objects.filter(origName__iexact = boardName, term = g.term)[0]
                g.parentGroup = pg
                print '%s -> %s' % (g.origName, g.name)
            except:
                pass

            # force presave commit hook to regenerate proper slug
            g.save()

    for g in Group.objects.all():
        g.slug = ''
        g.save()

def makeName(str):
    """
    It gets string in form 'Josef Karel Novak' the first word 'Josef' is
    stripped and it is returned as a name, the rest of the string is returned
    as a surname.
    """
    name = ''
    surname = ''
    s = str.split()
    name = s[0]
    for i in s[1:]:
        surname += i + ' '

    return name.strip(), surname.strip()
        

def getGroup(orgId, term):
    """
    gets group object for given ID and term ID. if no group object exists, it
    is fetched from psp.cz and new record is created in DB.
    """
    retVal = None
    if len(orgId) > 0:
        try:
            retVal = Group.objects.get(groupId__exact = orgId, term = term) 
            print 'Known Group: %s (%s)' % (retVal.origName, retVal.groupId)
        except ObjectDoesNotExist:
            groupName = utils.fetcher.getOrgName(orgId, term.termId)
            print 'New Group: %s (%s) %s' % (groupName, orgId, term.termId)
            retVal = Group()
            retVal.term = term
            retVal.origName = groupName.strip()
            retVal.groupId = orgId.strip()
            retVal.save()

    return retVal

    
def getArea(areaId, term):
    """
    gets Area name for given area ID and term ID if area does not exists in DB,
    it is fetched from psp.cz site and new record is created in DB
    """
    retVal = None
    try:
        retVal = Division.objects.get(divId__exact = areaId, term = term)
        print 'Known Area: %s (%s)' % (retVal.name, retVal.divId)
    except ObjectDoesNotExist:
        retVal = Division()
        divName = utils.fetcher.getOrgName(areaId, term.termId)
        print 'New Area: %s (%s)' % (divName, areaId)
        retVal.term = term
        retVal.name = divName.strip()
        retVal.divId = areaId.strip()
        retVal.save()
        
    return retVal


def updatePersonObject(person, details):
    """
    updates object from details, which are
    fetched from psp.cz site
    """
    if len(details['homePage']) > 0:
        person.homePage = details['homePage'].strip()

    if len(details['regOffice']) > 0:
        person.regOffice = details['regOffice'].strip()

    if len(details['office']) > 0:
        person.office = details['office'].strip()

    if len(details['phone']) > 0:
        person.phone = details['phone'].strip()

    if len(details['email']) > 0:
        person.email = details['email'].strip()

    person.save()
    

def updateDetail(person, term, area, mpDetails):
    """
    creates detail object for person if it does not exist.
    person, term, area are django objects,
    mpDetails is dictionary 
    """
    try:
        detail = PersonDetail.objects.get(person = person, term = term, region = area)
        print 'Detail for %s %s is up to date' % (person.name, person.surname)
    except ObjectDoesNotExist:
        detail = PersonDetail()

        try:
            f = open(mpDetails['photoFile'], 'r')
            detail.photo.save(mpDetails['photoFile'], File(f), save = True)
        except:
            pass

        detail.term = term
        detail.person = person
        detail.region = area
        detail.save()
        print 'Created new detail record for %s %s' % (person.name, person.surname)


def updateGroups(term, mpDict):
    """
    creates parliamentary groups, which does not exist yet.  the termId is
    string with termId, mpDict are data from list of MPs found
    http://www.psp.cz/sqw/snem.sqw?P1=0&P2=0&l=cz
    """
    for b in mpDict['boardIds']:
        board = getGroup(b, term)
        if board != None:
            board.save()

    for c in mpDict['commIds']:
        committee = getGroup(c, term)
        if committee != None:
            committee.save()

    for d in mpDict['delegIds']:
        deleg = getGroup(d, term)
        if deleg != None:
            deleg.save()


def updateMemberships(person, term, memberships):
    """
    Creates membership record objects.
    """
    for m in memberships:
        g = getGroup(m['orgId'], term)
        if g == None:
            try:
                g = Group.objects.get(origName__iexact = m['orgName'], term = term)
            except ObjectDoesNotExist:
                print 'No group found for %s (%s)' % (m['orgName'], m['orgId'])
                g = Group()
                g.term = term
                g.origName = m['orgName']
                g.name = m['orgName']
                g.groupId = m['orgId']
                g.type = m['orgType']

        g.type = m['orgType']
        if len(g.origName) == 0:
            g.origName = m['orgName']
        g.save()

        start = datetime.datetime(
                year = m['start']['year'],
                month = m['start']['month'],
                day = m['start']['day']
            )
        print start
        membership = Membership.objects.filter(
                            group = g,
                            person = person,
                            post__iexact = m['post'],
                            start = start
                        )
        if len(membership) != 0:
            if len(membership) == 1:
                try:
                    end = datetime.datetime(
                            year = m['end']['year'],
                            month = m['end']['month'],
                            day = m['end']['day']
                        )
                    if membership[0].end != end:
                        membership[0].end = end
                        membership[0].save()
                except:
                    pass
            else:
                print 'Warning found more identical memberships for %s %s (%s)' % (
                    person.name,
                    person.surname,
                    person.mpId
                )
                for i in membership:
                    print '\t%s - %s - (%d %d %d - %d %d %d)' % (
                        i.group.origName,
                        i.post,
                        i.start.day,
                        i.start.month,
                        i.start.year,
                        i.end.day,
                        i.end.month,
                        i.end.year
                    )

        else:
            membership = Membership()
            if len(g.type) == 0:
                g.type = 'PARL_GROUP'
            membership.group = g
            membership.person = person
            membership.post = m['post']
            membership.start = start
            try:
                membership.end = datetime.datetime(
                                    year = m['end']['year'],
                                    month = m['end']['month'],
                                    day = m['end']['day']
                                )
            except:
                pass
            membership.save()

            
def updateMPs(mps, term):
    """
    It gets list of MPs fetched from
        http://www.psp.cz/sqw/snem.sqw?P1=0&P2=0&l=cz
    it walks through list and updates details for every MP
    """

    for mp in mps:
        p = None
        personRecord = utils.fetcher.fetchMemberByID(mp['pspId'].strip())
        name, surname = makeName(mp['name'])
        if name == 'Miroslava':
            import pdb;pdb.set_trace()

        try:
            p = Person.objects.get(mpId__exact = mp['pspId'].strip())
            p.birthDate = datetime.date(
                            year = personRecord['birthDate']['year'],
                            month = personRecord['birthDate']['month'],
                            day = personRecord['birthDate']['day']
                        )
            p.save()
            print 'Updating: %s %s (%s) [ %d-%d-%d ]' % (p.name, p.surname, p.mpId,
                     p.birthDate.day, p.birthDate.month, p.birthDate.year)
        except ObjectDoesNotExist:
            p = Person()
            p.name, p.surname = makeName(mp['name'])
            p.title = mp['titlePre']
            p.titleLast = mp['titlePost']
            p.birthDate = datetime.date(
                            year = personRecord['birthDate']['year'],
                            month = personRecord['birthDate']['month'],
                            day = personRecord['birthDate']['day']
                        )
            p.mpId = mp['pspId'].strip()
            p.save()
            print 'Creating: %s %s (%s)' % (p.name, p.surname, p.mpId)

        party = getGroup(mp['partyId'], term)
        party.type = 'KLUB'
        party.save()
        #
        # this will allow us to postprocess groups later (see fixGroups())
        # once fixing names will be done in code, we can remove it
        #
        updateGroups(term, mp)

        mpDetails = utils.fetcher.fetchMP(mp['pspId'], term.termId)
        if mpDetails != None:
            updatePersonObject(p, mpDetails)
            area = getArea(mp['areaId'], term)
            updateDetail(p, term, area, mpDetails)
            updateMemberships(p, term, mpDetails['membship'])


def updateMeetings(meetings, term):
    for m in meetings:
        try:
            meeting = Meeting.objects.get(
                            term = term,
                            meetingNo__iexact = m['meetingNo']
                        )
        except ObjectDoesNotExist:
            meeting = Meeting()
            meeting.meetingNo = m['meetingNo']
            meeting.name = m['name']
            meeting.url = 'http://www.psp.cz/sqw%s' % (m['url'])
            start = m['days'][0]
            end = m['days'][len( m['days']) - 1 ]
            meeting.start = datetime.datetime(
                                    year = start['year'],
                                    month = start['month'],
                                    day = start['day']
                                )
            meeting.end = datetime.datetime(
                                    year = end['year'],
                                    month = end['month'],
                                    day = end['day']
                                )
            meeting.term = term
            meeting.save()
            print 'Saving meeting: %s (%s)' % (meeting.name, term.termId)



def updateStats(term, result):
    """
    function updates stats for MP in given date range
    """
    mpStat = None
    try:
        mpStat = VoteStats.objects.get(
                    person = result.person,
                    term = term
                )
    except ObjectDoesNotExist:
        print 'Creating stat for %s %s' % (result.person.name, result.person.surname)
        mpStat = VoteStats()
        mpStat.person = result.person
        mpStat.term = term
        mpStat.votesAye = 0
        mpStat.votesNay = 0
        mpStat.refrains = 0
        mpStat.absences = 0
        mpStat.excused = 0
        

    if result.result == 'A':
        mpStat.votesAye += 1
    elif result.result == 'N':
        mpStat.votesNay += 1
    elif result.result == '0':
        mpStat.absences += 1
    elif result.result == 'Z':
        mpStat.refrains += 1
    elif result.result == 'M' or result.result == 'X':
        mpStat.excused += 1
    mpStat.save()


def updatePolls(term, meetings = None):
    if meetings == None:
        meetings = Meeting.objects.filter(term = term)

    for m in meetings:
        results = utils.fetcher.fetchResultsOverview(term.termId, m.meetingNo)
        for r in results:
            poll = None
            try:
                poll = Poll.objects.get(meeting = m, pollId__iexact = r['pollId'])
                print 'Poll %s already exists' % poll.name
            except ObjectDoesNotExist:
                poll = Poll()
                poll.meeting = m
                poll.pollId = r['pollId']
                poll.name = r['pollName']
                poll.verdict = r['pollResult']
                poll.resultsLink = r['pollUrl']
                poll.stenoLink = r['pollDetailsURL']
                poll.date = datetime.datetime(
                                year = r['pollDate']['year'],
                                month = r['pollDate']['month'],
                                day = r['pollDate']['day']
                            )
                poll.votesAye = 0
                poll.votesNay = 0
                poll.votesRefrain = 0
                poll.absences = 0
                poll.excused = 0
                print 'Saved poll: %s' % (poll.name,)
                poll.save()

            if poll.votesAye == 0 and\
                poll.votesNay == 0 and\
                poll.votesRefrain == 0 and\
                poll.excused == 0 and\
                poll.absences == 0:

                votes = utils.fetcher.fetchPollResult(term.termId, poll.pollId)

                for vote in votes:
                    person = None
                    try:
                        person = Person.objects.get(mpId__iexact = vote['pspId'])
                    except ObjectDoesNotExist:
                        continue

                    v = MpVote()
                    v.poll = poll
                    v.person = person
                    v.result = vote['result']
                    try:
                        v.save()
                    except IntegrityError:
                        # the vote for MP already exists, ignore it
                        continue
                    poll.bumpPoll(v.result)
                        
                poll.save()
                GroupStats.objects.createStats(poll)
                for gs in GroupStats.objects.filter(poll = poll):
                    x = gs.getResult()

            else:
                print 'Results already exists for %s' % (poll.name,)
        
            stateData = SiteStateData.objects.get(id = 1)
            stateData.recentUpdate = poll.date
            stateData.lastMeeting = m
            stateData.save()


def updateDataForTerm(term = None):
    def findNewMeetings():
        lastMeeting = SiteStateData.objects.get(id = 1).recentMeeting
        i = 0
        meetings = Meeting.objects.all()

        for m in meetings:
            if m == lastMeeting:
                break
            i += 1

        return meetings[i:]
    """
    updates data for given term. if no term is passed
    it processes all terms.
    """
    if term == None:
        try:
            terms = TermOfOffice.objects.all().order_by('id')
        except:
            print 'No terms are defined'
            return
        
        for term in terms:
            #mps = utils.fetcher.fetchPSPMembers(term.termId)
            #updateMPs(mps, term)
            meetings = utils.fetcher.fetchPSPMeetings(term.termId)
            updateMeetings(meetings, term)
            updatePolls(term)
    else:
        print 'Updating term: %s - %s'  % (term.yearStart, term.yearEnd)
        meetings = utils.fetcher.fetchPSPMeetings(term.termId)
        updateMeetings(meetings, term)
        recentUpdate = None
        try:
            siteStateData = SiteStateData.objects.get(id = 1)
            meetings = findNewMeetings()
            if len(meetings) == 0:
                print "no new meetings"
                return
        except ObjectDoesNotExist:
            pass
        
        #mps = utils.fetcher.fetchPSPMembers(term.termId)
        #updateMPs(mps, term)
        updatePolls(term, meetings)
        updateDayStats()


def updateDayStats():
    def getDateList():
        meetingDates = Poll.objects.getMeetingDates()
        retVal = []

        try:
            ds = MpDayStats.objects.all().order_by('-date')
            recentDate = ds[0].date
            i = 0
            for md in meetingDates:
                if md > recentDate:
                    retVal = meetingDates[i:]
                    print "Starting from: %d %d %d" % ( retVal[0].year,
                                                        retVal[0].month,
                                                        retVal[0].day
                                                    )
                    break
                i = i + 1

        except:
            print "Starting from the begining %d %d %d" % ( retVal[0].year,
                                                            retVal[0].month,
                                                            retVal[0].day
                                                        )

        return retVal


    dates = getDateList()

    for d in dates:
        print "Counting stats for: %s" % d.strftime("%d. %B %Y")
        MpDayStats.objects.createStatsForDay(d)
        for stat in MpDayStats.objects.all().filter(date = d):
            stat.save()

        print "done\n"

    return 0

def update():
    lastTerm = None

    terms = TermOfOffice.objects.all().order_by('id')
    if len(terms) > 0:
        lastTerm = terms[len(terms) - 1]

    webTerms = utils.fetcher.fetchTerms()
    webTerms.reverse()

    if lastTerm == None:
        # DB is empty
        for webTerm in webTerms:
            term = TermOfOffice()
            term.yearStart = webTerm['yearStart']
            term.yearEnd = webTerm['yearEnd']
            term.termId = webTerm['termId']
            term.save()
            print 'Created Term: %s' % term.slug
            updateDataForTerm(term)
    else:
        # Find last term in DB to start update
        i = 0
        for webTerm in webTerms:
            if webTerm['termId'] == lastTerm.termId:
                break
            i += 1

        print "Starting at %s - %s" % (lastTerm.yearStart, lastTerm.yearEnd)
        updateDataForTerm(lastTerm)
        i += 1

        for webTerm in webTerms[i:]:
            term = TermOfOffice()
            term.yearStart = webTerm['yearStart']
            term.yearEnd = webTerm['yearEnd']
            term.termId = webTerm['termId']
            term.save()
            print 'Created Term: %s' % term.slug
            updateDataForTerm(term)


def fetchMissingResults():
    def getMissingResults():
        retVal = []
        for p in Poll.objects.filter(
                                vote_aye = 0,
                                vote_nay = 0,
                                refrain = 0,
                                absentees = 0,
                                excused = 0
                            ):
            if MpVote.objects.filter(poll = p).count() == 0:
                retVal.append(p)

        return retVal

    def savePollResults(poll):
        votes = utils.fetcher.fetchPollResult(poll.meeting.term.termId, poll.pollId)
        term = poll.meeting.term
        print 'Fetched votes for %s' % (poll.name)
        for vote in votes:
            person = None
            try:
                person = Person.objects.get(mpId__iexact = vote['pspId'])
            except ObjectDoesNotExist:
                continue

            """
            We are not going to look up single vote here avoiding creating
            duplicate votes here. we are going to save all or none.
            """
            v = MpVote()
            v.poll = poll
            v.person = person
            v.result = vote['result']
            try:
                v.save()
            except IntegrityError:
                # the vote for MP already exists, ignore it
                continue

            poll.bumpPoll(v.result)                 
            poll.save()

    for p in getMissingResults():
        savePollResults(p)
        GroupStats.objects.createStats(poll)
    
def main():
    update()
    fixNames()
    fixOrgs()
    #fetchMissingResults()
#   meetings = Meeting.objects.filter(meetingNo__exact = '53')
#   for m in meetings:
#       print (m.start, ' - ', m.end)
#   
#   m = meetings[0]
#   polls = Poll.objects.filter(meeting = m)
#   
#   g = Group.objects.get(groupId__exact = '-1')
#   for p in polls:
#       members = Membership.objects.filter(start__gte='2002-01-01', end__lte='2006-06-30', group = g)
#       for member in members:
#           votes = MpVote.objects.filter(poll = p, person = member.person)
#           if votes:
#               if len(votes) > 1:
#                   print p.name
#                   delThem = {}
#
#                   r = votes[0].result
#                   doDelete = True
#                   for v in votes:
#                       print '%d: %s %s' % (v.id, member.person.name, member.person.surname)
#                       delThem[v.id] = v
#                       if v.result != r:
#                           doDelete = False
#
#                   if doDelete:
#                       print 'going to normalize result'
#                       list = delThem.keys()
#                       list.sort()
#                       for i in list[0:len(list) - 1]:
#                           print 'deleting: %d' % delThem[i].id
#                           delThem[i].delete()

if __name__ == '__main__':
    main()
    sys.exit()
